import company_find
import time
import os
import json
import collections
import io_util
from nlp_util import clean


# save_json('company.json',{k:[] for k in companys})
def get_company_and_counter():
    d = io_util.load_json('company.json')  # 放入公司
    companys = [u for u in d]  #
    print('companys vocab', len(companys))
    counter = collections.Counter(companys)  # 为了取top k 个公司名进行处理
    a = counter.most_common(100000)  #
    last_word = a[-1][0]
    print(counter[last_word])
    return companys, counter


def run_start(counter):
    a = counter.most_common(100000)  #
    companys = [word for word, freq in a]
    companys = [clean(c) for c in companys]
    companys = [c for c in companys if len(c) > 1]
    # companys = sorted(companys,key=lambda x:len(x))
    print('company', len(companys))
    # companys = [c for c in companys if len(c)< 14]
    print(companys[-1])
    res_d = {}
    kws = [c for c in companys if c not in ['集团', "工作室", "猎头公司", '中外合资'] and len(c) >= 4]
    if os.path.exist('freqbaike_start2.json'):
        res_d = io_util.load_json('freqbaike_start2.json')
    # vocab = set(freq.keys())
    # print('inter',len(set(vocab).intersection(set(kws))))
    run = True
    # print(kws[:100])
    if run:
        s = time.time()
        print(s)
        for i in range(len(kws)):
            print(i)
            if i % 100 == 0:
                # 批量请求 100
                sub_ws = kws[i:i + 100]
                if all([u in res_d for u in sub_ws]):
                    continue
                # sub_ws = [w for w in kws if len(freq.get(w,[]))==0]
                print('sub_ws')
                sk = company_find.SearchInformation(keywords=sub_ws, processes=10)
                res = sk.thread_req()
                while res is None:
                    res = sk.thread_req()
                # print(res)
                print('thread req')
                res_d.update({k: v for k, v in zip(sub_ws, res)})
            if i % 1000 == 0:
                # 存储本地
                print(i, int(time.time() - s))
                io_util.save_json('freqbaike88.json', res_d)
                # io_util.save_cache('freqbaike.pk', res_d)
            if i % 2000 == 0:
                # 存储本地
                io_util.save_json('freqbaike88_2000.json', res_d)
                # io_util.save_cache('freqbaike2000.pk',res_d)
